{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM-Inception Text Classification with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[i] Loaded Parameters:\n",
      " 40000 35 0.2 200 \n",
      " dataset/glove/glove.twitter.27B.200d.txt\n",
      "[i] Importing Modules...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[+] Using Keras version 2.1.4\n",
      "[i] Finished Importing Modules\n",
      "[i] . Reading from csv file...Done!\n",
      "[i] Found 30447 unique tokens.\n",
      "[+] Shape of data tensor: (40000, 30)\n",
      "[+] Shape of label tensor: (40000, 4)\n",
      "[+] Number of entries in each category:\n",
      "[+] Training:\n",
      " [ 7708. 10867. 12305.  1120.]\n",
      "[+] Validation:\n",
      " [1936. 2757. 2994.  313.]\n",
      "[i] Loading GloVe from: dataset/glove/glove.twitter.27B.200d.txt ...Done.\n",
      "[+] Proceeding with Embedding Matrix...Completed!\n",
      "Finished running setup.\n"
     ]
    }
   ],
   "source": [
    "%run Setup.ipynb\n",
    "%run ExtraFunctions.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load embedding matrix into an `Embedding` layer. Toggle `trainable=False` to prevent the weights from being updated during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM + Inception Structure\n",
    "[Reference](https://github.com/richliao/textClassifier), [LTSM](http://colah.github.io/posts/2015-08-Understanding-LSTMs/), [CNN Source](https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html) and [Notes](http://www.wildml.com/2015/11/understanding-convolutional-neural-networks-for-nlp/)\n",
    "\n",
    "Deeper CNN as described in [CNN for Sentence Classification](http://www.aclweb.org/anthology/D14-1181) (Yoon Kim, 2014), multiple filters have been applied. This can be implemented using Keras `Merge` Layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "\n",
    "l_lstm1 = Bidirectional(LSTM(4,dropout=0.3,recurrent_dropout=0.3,return_sequences=True))(embedded_sequences)\n",
    "\n",
    "# inception module\n",
    "inception, filter_sizes = [], [3,4,6,7]\n",
    "for fsz in filter_sizes:\n",
    "    l_conv_i1 = Conv1D(filters=12,kernel_size=1,\n",
    "                    activation='relu',)(l_lstm1)\n",
    "    l_conv_i2 = Conv1D(filters=12,kernel_size=fsz,\n",
    "                       activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_conv_i1)\n",
    "    inception.append(l_conv_i2)\n",
    "l_pool_inc = MaxPooling1D(4)(l_lstm1)\n",
    "l_conv_inc = Conv1D(filters=12,kernel_size=1,\n",
    "                    activation='relu',kernel_regularizer=regularizers.l2(0.02))(l_pool_inc)\n",
    "inception.append(l_conv_inc)\n",
    "\n",
    "l_merge = Concatenate(axis=1)(inception)\n",
    "l_pool1 = MaxPooling1D(3)(l_merge)\n",
    "l_drop1 = Dropout(0.4)(l_pool1)\n",
    "l_flat = Flatten()(l_drop1)\n",
    "l_dense = Dense(16, activation='relu',kernel_regularizer=regularizers.l2(0.02))(l_flat)\n",
    "\n",
    "preds = Dense(4, activation='softmax')(l_dense)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = Model(sequence_input, preds)\n",
    "adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)\n",
    "lr_metric = get_lr_metric(adadelta)\n",
    "#keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=adadelta,\n",
    "              metrics=['acc', lr_metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=16, write_grads=True , write_graph=True)\n",
    "model_checkpoints = callbacks.ModelCheckpoint(\"checkpoint.h5\", monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)\n",
    "lr_schedule = callbacks.LearningRateScheduler(initial_boost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 30)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 30, 200)      6089600     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 30, 8)        6560        embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 30, 12)       108         bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 30, 12)       108         bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 30, 12)       108         bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)               (None, 30, 12)       108         bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)  (None, 7, 8)         0           bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 28, 12)       444         conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 27, 12)       588         conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 25, 12)       876         conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_8 (Conv1D)               (None, 24, 12)       1020        conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_9 (Conv1D)               (None, 7, 12)        108         max_pooling1d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 111, 12)      0           conv1d_2[0][0]                   \n",
      "                                                                 conv1d_4[0][0]                   \n",
      "                                                                 conv1d_6[0][0]                   \n",
      "                                                                 conv1d_8[0][0]                   \n",
      "                                                                 conv1d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 37, 12)       0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 37, 12)       0           max_pooling1d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 444)          0           dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 16)           7120        flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 4)            68          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 6,106,816\n",
      "Trainable params: 6,106,816\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Training Progress:\n",
      "Train on 32000 samples, validate on 8000 samples\n",
      "Epoch 1/200\n",
      "32000/32000 [==============================] - 76s 2ms/step - loss: 1.2101 - acc: 0.4198 - lr: 6.0000 - val_loss: 1.1043 - val_acc: 0.5136 - val_lr: 6.0000\n",
      "Epoch 2/200\n",
      "32000/32000 [==============================] - 75s 2ms/step - loss: 1.0598 - acc: 0.5478 - lr: 1.0000 - val_loss: 1.0496 - val_acc: 0.5531 - val_lr: 1.0000\n",
      "Epoch 3/200\n",
      "32000/32000 [==============================] - 86s 3ms/step - loss: 1.0390 - acc: 0.5603 - lr: 1.0000 - val_loss: 1.0351 - val_acc: 0.5649 - val_lr: 1.0000\n",
      "Epoch 4/200\n",
      "32000/32000 [==============================] - 87s 3ms/step - loss: 1.0223 - acc: 0.5752 - lr: 1.0000 - val_loss: 1.0314 - val_acc: 0.5695 - val_lr: 1.0000\n",
      "Epoch 5/200\n",
      "32000/32000 [==============================] - 87s 3ms/step - loss: 1.0114 - acc: 0.5826 - lr: 1.0000 - val_loss: 1.0224 - val_acc: 0.5729 - val_lr: 1.0000\n",
      "Epoch 6/200\n",
      "32000/32000 [==============================] - 85s 3ms/step - loss: 1.0055 - acc: 0.5837 - lr: 1.0000 - val_loss: 1.0159 - val_acc: 0.5734 - val_lr: 1.0000\n",
      "Epoch 7/200\n",
      "32000/32000 [==============================] - 86s 3ms/step - loss: 0.9965 - acc: 0.5904 - lr: 1.0000 - val_loss: 1.0146 - val_acc: 0.5755 - val_lr: 1.0000\n",
      "Epoch 8/200\n",
      "32000/32000 [==============================] - 87s 3ms/step - loss: 0.9911 - acc: 0.5944 - lr: 1.0000 - val_loss: 1.0196 - val_acc: 0.5773 - val_lr: 1.0000\n",
      "Epoch 9/200\n",
      "32000/32000 [==============================] - 83s 3ms/step - loss: 0.9834 - acc: 0.5975 - lr: 1.0000 - val_loss: 1.0191 - val_acc: 0.5698 - val_lr: 1.0000\n",
      "Epoch 10/200\n",
      "32000/32000 [==============================] - 87s 3ms/step - loss: 0.9754 - acc: 0.6020 - lr: 1.0000 - val_loss: 1.0108 - val_acc: 0.5776 - val_lr: 1.0000\n",
      "Epoch 11/200\n",
      "32000/32000 [==============================] - 86s 3ms/step - loss: 0.9706 - acc: 0.6063 - lr: 1.0000 - val_loss: 1.0110 - val_acc: 0.5768 - val_lr: 1.0000\n",
      "Epoch 12/200\n",
      "32000/32000 [==============================] - 87s 3ms/step - loss: 0.9680 - acc: 0.6063 - lr: 1.0000 - val_loss: 1.0206 - val_acc: 0.5774 - val_lr: 1.0000\n",
      "Epoch 13/200\n",
      "32000/32000 [==============================] - 86s 3ms/step - loss: 0.9636 - acc: 0.6107 - lr: 1.0000 - val_loss: 1.0238 - val_acc: 0.5731 - val_lr: 1.0000\n",
      "Epoch 14/200\n",
      "32000/32000 [==============================] - 85s 3ms/step - loss: 0.9588 - acc: 0.6149 - lr: 1.0000 - val_loss: 1.0079 - val_acc: 0.5769 - val_lr: 1.0000\n",
      "Epoch 15/200\n",
      "32000/32000 [==============================] - 84s 3ms/step - loss: 0.9532 - acc: 0.6169 - lr: 1.0000 - val_loss: 1.0103 - val_acc: 0.5791 - val_lr: 1.0000\n",
      "Epoch 16/200\n",
      "32000/32000 [==============================] - 84s 3ms/step - loss: 0.9505 - acc: 0.6185 - lr: 1.0000 - val_loss: 1.0157 - val_acc: 0.5755 - val_lr: 1.0000\n",
      "Epoch 17/200\n",
      "32000/32000 [==============================] - 84s 3ms/step - loss: 0.9455 - acc: 0.6191 - lr: 1.0000 - val_loss: 1.0125 - val_acc: 0.5804 - val_lr: 1.0000\n",
      "Epoch 18/200\n",
      "32000/32000 [==============================] - 85s 3ms/step - loss: 0.9414 - acc: 0.6213 - lr: 1.0000 - val_loss: 1.0188 - val_acc: 0.5811 - val_lr: 1.0000\n",
      "Epoch 19/200\n",
      "32000/32000 [==============================] - 84s 3ms/step - loss: 0.9365 - acc: 0.6263 - lr: 1.0000 - val_loss: 1.0156 - val_acc: 0.5795 - val_lr: 1.0000\n",
      "Epoch 20/200\n",
      "32000/32000 [==============================] - 84s 3ms/step - loss: 0.9338 - acc: 0.6272 - lr: 1.0000 - val_loss: 1.0172 - val_acc: 0.5795 - val_lr: 1.0000\n",
      "Epoch 21/200\n",
      "32000/32000 [==============================] - 84s 3ms/step - loss: 0.9310 - acc: 0.6292 - lr: 1.0000 - val_loss: 1.0269 - val_acc: 0.5770 - val_lr: 1.0000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22/200\n",
      "32000/32000 [==============================] - 85s 3ms/step - loss: 0.9264 - acc: 0.6315 - lr: 1.0000 - val_loss: 1.0215 - val_acc: 0.5764 - val_lr: 1.0000\n",
      "Epoch 23/200\n",
      "32000/32000 [==============================] - 84s 3ms/step - loss: 0.9230 - acc: 0.6343 - lr: 1.0000 - val_loss: 1.0210 - val_acc: 0.5756 - val_lr: 1.0000\n",
      "Epoch 24/200\n",
      "10250/32000 [========>.....................] - ETA: 57s - loss: 0.9092 - acc: 0.6455 - lr: 1.0000"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-7b111878741f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m model_log = model.fit(x_train, y_train, validation_data=(x_val, y_val),\n\u001b[1;32m      5\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m           callbacks=[tensorboard, lr_schedule])\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1710\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1711\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1712\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1713\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1714\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1233\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1236\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1237\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2473\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2474\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2475\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2476\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1126\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1127\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1128\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1129\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1130\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1344\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1345\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1346\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1348\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1350\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1327\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1328\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1329\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1331\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.summary()\n",
    "model.save('ltsm-inception.h5')\n",
    "print(\"Training Progress:\")\n",
    "model_log = model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "          epochs=200, batch_size=50,\n",
    "          callbacks=[tensorboard, lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('ltsm-inception.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(model_log.history).to_csv(\"history-inception.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
